Exploratory Text Analytics Final Project¶
Arti Patel (ap8qk@virginia.edu)
Sudeepti Surapaneni(ss9ud@virginia.edu)
Adonis Lu (ayl3yq@virginia.edu)
DS 5001
29 April 2020
Import¶
In [1]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
%matplotlib inline
from glob import glob
import re
import nltk
import plotly_express as px
In [2]:
# CONFIG
OHCO = ['book_id', 'chap_num', 'para_num', 'sent_num', 'token_num']
epub_dir = 'epubs_'
Inspect¶
Since Project Gutenberg texts vary widely in their markup, we define our chunking patterns by hand.
In [3]:
roman = '[IVXLCM]+'
caps = "[A-Z';, -]+"
chap_pats = {
2787: {
'start_line': 21,
'end_line': 11235,
#'chapter': re.compile("^\s*CHAPTER\s+{}\.\s*$".format(roman))
'chapter': re.compile("^\s*CHAPTER\s+{}\. .*$".format(roman))
},
2726: {
'start_line': 21,
'end_line': 11235,
#'chapter': re.compile("^\s*CHAPTER\s+{}\.\s*$".format(roman))
'chapter': re.compile("^Chapter\s+\d+.+$")
},
2786: {
'start_line': 21,
'end_line': 11235,
'chapter': re.compile("^\s*Chapter\s+{}\. .*$".format(roman))
},
514: {
'start_line': 21,
'end_line': 11235,
'chapter': re.compile('^CHAPTER+\s*{}\s*$'.format(caps))
},
3499: {
'start_line': 21,
'end_line': 10256,
'chapter': re.compile("^Chapter\s+\d+.+$")
},
2788: {
'start_line': 21,
'end_line': 11620,
#'chapter': re.compile('^CHAPTER+\s*{}\s*$'.format(roman))
'chapter': re.compile("^\s*CHAPTER\s+{}\. .*$".format(roman))
},
2804: {
'start_line': 21,
'end_line': 11235,
'chapter': re.compile("^Chapter\s+\d+.+$")
},
3795: {
'start_line': 21,
'end_line': 11235,
'chapter': re.compile('^CHAPTER+\s*{}\s*$'.format(roman))
}
}
Register and Chunk¶
In [4]:
def acquire_epubs(epub_list, chap_pats, OHCO=OHCO):
my_lib = []
my_doc = []
for epub_file in epub_list:
# Get PG ID from filename
book_id = int(epub_file.split('-')[-1].split('.')[0].replace('pg',''))
print("BOOK ID", book_id)
# Import file as lines
lines = open(epub_file, 'r', encoding='utf-8-sig').readlines()
df = pd.DataFrame(lines, columns=['line_str'])
df.index.name = 'line_num'
df.line_str = df.line_str.str.strip()
df['book_id'] = book_id
# FIX CHARACTERS TO IMPROVE TOKENIZATION
df.line_str = df.line_str.str.replace('—', ' — ')
df.line_str = df.line_str.str.replace('-', ' - ')
# Get book title and put into LIB table -- note problems, though
book_title = re.sub(r"The Project Gutenberg eBook( of|,) ", "", df.loc[0].line_str, flags=re.IGNORECASE)
#book_title = re.sub(r"The Project Gutenberg eBook( of|,) ", "", df.loc[1].line_str, flags=re.IGNORECASE)
book_title = re.sub(r"Project Gutenberg's ", "", book_title, flags=re.IGNORECASE)
# Remove cruft
a = chap_pats[book_id]['start_line'] - 1
b = chap_pats[book_id]['end_line'] + 1
df = df.iloc[a:b]
# Chunk by chapter
chap_lines = df.line_str.str.match(chap_pats[book_id]['chapter'])
chap_nums = [i+1 for i in range(df.loc[chap_lines].shape[0])]
df.loc[chap_lines, 'chap_num'] = chap_nums
df.chap_num = df.chap_num.ffill()
# Clean up
df = df[~df.chap_num.isna()] # Remove chapter heading lines
df = df.loc[~chap_lines] # Remove everything before Chapter 1
df['chap_num'] = df['chap_num'].astype('int')
# Group -- Note that we exclude the book level in the OHCO at this point
df = df.groupby(OHCO[1:2]).line_str.apply(lambda x: '\n'.join(x)).to_frame() # Make big string
# Split into paragrpahs
df = df['line_str'].str.split(r'\n\n+', expand=True).stack().to_frame().rename(columns={0:'para_str'})
df.index.names = OHCO[1:3] # MAY NOT BE NECESSARY UNTIL THE END
df['para_str'] = df['para_str'].str.replace(r'\n', ' ').str.strip()
df = df[~df['para_str'].str.match(r'^\s*$')] # Remove empty paragraphs
# Set index
df['book_id'] = book_id
df = df.reset_index().set_index(OHCO[:3])
# Register
my_lib.append((book_id, book_title, epub_file))
my_doc.append(df)
docs = pd.concat(my_doc)
library = pd.DataFrame(my_lib, columns=['book_id', 'book_title', 'book_file']).set_index('book_id')
print("Done.")
return library, docs
In [5]:
epubs = [epub for epub in sorted(glob(epub_dir+'/*.txt'))]
LIB, DOC = acquire_epubs(epubs, chap_pats)
LIB_arti = pd.read_csv('LIB_arti.csv')
DOC_arti = pd.read_csv('DOC_arti.csv')
DOC_arti = DOC_arti.rename(columns = {'story_num':'chap_num'})
DOC_arti = DOC_arti.set_index(['book_id','chap_num','para_num'])
LIB['book_id'] = LIB.index
LIB = LIB.append(LIB_arti)
DOC = DOC.append(DOC_arti)
LIB.index = LIB['book_id']
In [6]:
LIB
Out[6]:
In [7]:
DOC.sample(10)
Out[7]:
Tokenize and Annotate¶
We use NLTK this time. Note that this process takes some time, mainly because the NLTK functions are not optimized for dataframes.
Note that we can choose between tokenizers. NLTK offers a variety of them. Here is a list.
In [8]:
def tokenize(doc_df, OHCO=OHCO, remove_pos_tuple=False, ws=False):
# Paragraphs to Sentences
df = doc_df.para_str\
.apply(lambda x: pd.Series(nltk.sent_tokenize(x)))\
.stack()\
.to_frame()\
.rename(columns={0:'sent_str'})
# Sentences to Tokens
# Local function to pick tokenizer
def word_tokenize(x):
if ws:
s = pd.Series(nltk.pos_tag(nltk.WhitespaceTokenizer().tokenize(x)))
else:
s = pd.Series(nltk.pos_tag(nltk.word_tokenize(x))) # Discards stuff in between
return s
df = df.sent_str\
.apply(word_tokenize)\
.stack()\
.to_frame()\
.rename(columns={0:'pos_tuple'})
# Grab info from tuple
df['pos'] = df.pos_tuple.apply(lambda x: x[1])
df['token_str'] = df.pos_tuple.apply(lambda x: x[0])
if remove_pos_tuple:
df = df.drop('pos_tuple', 1)
# Add index
df.index.names = OHCO
return df
In [9]:
%%time
TOKEN = tokenize(DOC, ws=False)
In [10]:
TOKEN.head()
Out[10]:
In [11]:
TOKEN[TOKEN.pos.str.match('^NNP')]
Out[11]:
Reduce¶
Extract a vocabulary from the TOKEN table
In [12]:
TOKEN['term_str'] = TOKEN['token_str'].str.lower().str.replace('[\W_]', '')
In [13]:
VOCAB = TOKEN.term_str.value_counts().to_frame()\
.rename(columns={'index':'term_str', 'term_str':'n'})\
.sort_index().reset_index().rename(columns={'index':'term_str'})
VOCAB.index.name = 'term_id'
In [14]:
VOCAB['num'] = VOCAB.term_str.str.match("\d+").astype('int')
In [15]:
VOCAB
Out[15]:
Annotate (VOCAB)¶
Add Stopwords¶
We use NLTK's built in stopword list for English. Note that we can add and subtract from this list, or just create our own list and keep it in our data model.
In [16]:
sw = pd.DataFrame(nltk.corpus.stopwords.words('english'), columns=['term_str'])
sw = sw.reset_index().set_index('term_str')
sw.columns = ['dummy']
sw.dummy = 1
In [17]:
sw.sample(10)
Out[17]:
In [18]:
VOCAB['stop'] = VOCAB.term_str.map(sw.dummy)
VOCAB['stop'] = VOCAB['stop'].fillna(0).astype('int')
In [19]:
VOCAB[VOCAB.stop == 1].sample(10)
Out[19]:
Add Stems¶
In [20]:
from nltk.stem.porter import PorterStemmer
stemmer1 = PorterStemmer()
VOCAB['stem_porter'] = VOCAB.term_str.apply(stemmer1.stem)
In [21]:
VOCAB.sample(10)
VOCAB.index.name = 'term_id'
In [22]:
MFCB = TOKEN.groupby(['term_str','pos']).pos.count().unstack().idxmax(1).to_frame()
VOCAB = pd.merge(VOCAB,MFCB, left_on='term_str', right_on = 'term_str')
VOCAB = VOCAB.rename(columns = {0:'pos_max'})
Save¶
In [23]:
VOCAB.sample(5)
VOCAB.index.name = 'term_id'
In [24]:
DOC.to_csv('DOC.csv')
LIB.to_csv('LIBRARY.csv')
VOCAB.to_csv('VOCAB.csv')
TOKEN.to_csv('TOKEN.csv')
TFIDF¶
In [5]:
count_method = 'n' # 'c' or 'n' # n = n tokens, c = distinct token (term) count
tf_method = 'sum' # sum, max, log, double_norm, raw, binary
tf_norm_k = .5 # only used for double_norm
idf_method = 'standard' # standard, max, smooth
gradient_cmap = 'YlGnBu' # YlGn, GnBu, YlGnBu; For tables; see https://matplotlib.org/3.1.0/tutorials/colors/colormaps.html
In [6]:
OHCO = ['book_id', 'chap_num', 'para_num', 'sent_num', 'token_num']
SENTS = OHCO[:4]
PARAS = OHCO[:3]
CHAPS = OHCO[:2]
BOOKS = OHCO[:1]
In [7]:
bag = CHAPS
In [8]:
pd.__version__
Out[8]:
In [9]:
sns.set()
%matplotlib inline
Prepare the data¶
Import tables¶
Bring the the tables we created last time.
In [10]:
%%time
LIB = pd.read_csv("LIBRARY.csv").set_index(BOOKS)
TOKEN = pd.read_csv('TOKEN.csv').set_index(OHCO)
VOCAB = pd.read_csv('VOCAB.csv').set_index('term_id')
# DOC = pd.read_csv(data_dir + "DOC.csv").set_index(PARAS)
In [11]:
LIB = LIB[['book_title', 'book_file']]
LIB
Out[11]:
In [12]:
VOCAB.head()
Out[12]:
In [13]:
VOCAB = VOCAB[~VOCAB.term_str.isna()]
In [14]:
VOCAB.sample(5)
Out[14]:
In [15]:
TOKEN.head()
Out[15]:
In [16]:
TOKEN = TOKEN[~TOKEN.term_str.isna()]
In [17]:
TOKEN.head()
Out[17]:
In [18]:
# DOC.head()
Add term_id to TOKEN table¶
We need to do this to combine the VOCAB and TOKEN tables more efficiently. Note, we could have done this in the previous lab.
We use .map() because TOKEN and VOCAB do not share an index at this time.
In [20]:
TOKEN['term_id'] = TOKEN.term_str.map(VOCAB.reset_index().set_index('term_str').term_id)
In [21]:
TOKEN.head()
Out[21]:
Add Max POS to VOCAB¶
Just in case it's not there. It's easy now that we have a share feature -- term_id -- between VOCAB and TOKEN.
Regarding collisions when using .idxmax(), the documentation says "If multiple values equal the maximum, the first row label with that value is returned."
In [22]:
# Demo
# TOKEN.groupby(['term_id', 'pos']).pos.count()
# TOKEN.groupby(['term_id', 'pos']).pos.count().unstack()
# TOKEN.groupby(['term_id', 'pos']).pos.count().unstack().idxmax(1)
In [23]:
VOCAB['pos_max'] = TOKEN.groupby(['term_id', 'pos']).pos.count().unstack().idxmax(1)
In [24]:
VOCAB.sample(5)
Out[24]:
Compare POS Stats in TOKEN and VOCAB¶
Pause and look at distribution of POS tags. The POS table could become part of your data model (analytical edition) if you were interested in studying POS tags.
In [25]:
POS = TOKEN.pos.value_counts().to_frame().rename(columns={'pos':'n'})
POS.index.name = 'pos_id'
In [26]:
POS.sort_values('n').plot.bar(y='n', figsize=(15,5), rot=45);
Zipf's Law¶
$f \propto \frac{1}{r} $
$k = fr$
Add Term Rank to VOCAB¶
In [27]:
if 'term_rank' not in VOCAB.columns:
VOCAB = VOCAB.sort_values('n', ascending=False).reset_index()
VOCAB.index.name = 'term_rank'
VOCAB = VOCAB.reset_index()
VOCAB = VOCAB.set_index('term_id')
VOCAB['term_rank'] = VOCAB['term_rank'] + 1
In [28]:
VOCAB.head()
Out[28]:
Alternate Rank¶
The term_rank as defined above assigns different ranks to words with the same frequency, which occurs in the long tail, e.g. with words that appear once.
This measure groups words by term count.
In [29]:
new_rank = VOCAB.n.value_counts()\
.sort_index(ascending=False).reset_index().reset_index()\
.rename(columns={'level_0':'term_rank2', 'index':'n', 'n':'nn'})\
.set_index('n')
In [30]:
new_rank.head()
Out[30]:
In [31]:
VOCAB['term_rank2'] = VOCAB.n.map(new_rank.term_rank2) + 1
In [32]:
VOCAB.head()
Out[32]:
In [33]:
VOCAB['p'] = VOCAB.n / VOCAB.shape[0]
Compute Zipf's K¶
In [34]:
VOCAB['zipf_k'] = VOCAB.n * VOCAB.term_rank
VOCAB['zipf_k2'] = VOCAB.n * VOCAB.term_rank2
VOCAB['zipf_k3'] = VOCAB.p * VOCAB.term_rank2
In [35]:
VOCAB.describe().T
Out[35]:
Words with low k¶
In [36]:
VOCAB[VOCAB.zipf_k <= VOCAB.zipf_k.quantile(.1)].sort_values('zipf_k3', ascending=True).head()
Out[36]:
Words with high k¶
In [37]:
VOCAB[VOCAB.zipf_k >= VOCAB.zipf_k.quantile(.9)].sort_values('zipf_k3', ascending=False).head()
Out[37]:
Visualize¶
Histogram of Zipf K¶
In [38]:
# px.histogram(VOCAB, 'zipf_k', marginal='box')
In [39]:
# px.histogram(VOCAB, 'zipf_k2', marginal='box')
In [40]:
# px.histogram(VOCAB, 'zipf_k3', marginal='box')
Rank and N¶
In [41]:
VSAMP1 = VOCAB[['n','term_rank','zipf_k','term_str','pos_max']]
VSAMP2 = VOCAB[['n','term_rank2','zipf_k3']].drop_duplicates()
In [42]:
# px.scatter(VSAMP1, x='term_rank', y='n', log_y=False, log_x=False, hover_name='term_str', color='pos_max')
In [43]:
# px.scatter(VSAMP2, x='term_rank2', y='n', log_y=False, log_x=False)
In [44]:
# px.scatter(VSAMP1, x='term_rank', y='n', log_y=True, log_x=True, hover_name='term_str', color='pos_max')
In [45]:
# px.scatter(VSAMP2, x='term_rank2', y='n', log_y=True, log_x=True)
Demo Rank Index¶
In [46]:
rank_index = [1, 2, 3, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000]
In [47]:
demo = VOCAB.loc[VOCAB.term_rank.isin(rank_index), ['term_str', 'term_rank', 'n', 'zipf_k', 'pos_max']]
In [48]:
demo.style.background_gradient(cmap=gradient_cmap, high=.5)
Out[48]:
VOCAB Entropy¶
Compute P of VOCAB¶
This is the prior, or marginal, probability of a term.
In [49]:
%%time
VOCAB['p2'] = VOCAB.n / VOCAB.n.sum()
Compute Entropy of VOCAB¶
In [50]:
VOCAB['h'] = VOCAB.p2 * np.log2(1/VOCAB.p2) # Self entropy of each word
H = VOCAB.h.sum()
N_v = VOCAB.shape[0]
H_max = np.log2(N_v)
R = round(1 - (H/H_max), 2) * 100
In [51]:
print("H \t= {}\nH_max \t= {}\nR \t= {}%".format(H, H_max, int(R)))
BOW¶
In [52]:
BOW = TOKEN.groupby(bag+['term_id']).term_id.count()\
.to_frame().rename(columns={'term_id':'n'})
In [53]:
BOW['c'] = BOW.n.astype('bool').astype('int')
In [54]:
BOW.head(10)
BOW.to_csv('BOW.csv')
Document-Term Matrix¶
We create a document-term count matrix. Note that we can create a matrix for any of the features in BOW. Also, see how the OHCO helps us distinguish between features and observation identity.
Note, these operations are slower than using groupby().
Create Count Matrix¶
In [55]:
%%time
DTCM = BOW[count_method].unstack().fillna(0).astype('int')
In [56]:
DTCM.head()
Out[56]:
Compute TF¶
We could also compute that using BOW.groupby().
In [57]:
%%time
print('TF method:', tf_method)
if tf_method == 'sum':
TF = DTCM.T / DTCM.T.sum()
elif tf_method == 'max':
TF = DTCM.T / DTCM.T.max()
elif tf_method == 'log':
TF = np.log10(1 + DTCM.T)
elif tf_method == 'raw':
TF = DTCM.T
elif tf_method == 'double_norm':
TF = DTCM.T / DTCM.T.max()
TF = tf_norm_k + (1 - tf_norm_k) * TF[TF > 0] # EXPLAIN; may defeat purpose of norming
elif tf_method == 'binary':
TF = DTCM.T.astype('bool').astype('int')
TF = TF.T
In [58]:
TF.head()
Out[58]:
Compute DF¶
In [59]:
%%time
DF = DTCM[DTCM > 0].count()
In [60]:
DF.head()
Out[60]:
Compute IDF¶
In [61]:
N = DTCM.shape[0]
In [62]:
print('IDF method:', idf_method)
if idf_method == 'standard':
IDF = np.log10(N / DF)
elif idf_method == 'max':
IDF = np.log10(DF.max() / DF)
elif idf_method == 'smooth':
IDF = np.log10((1 + N) / (1 + DF)) + 1 # Correct?
Compute TFIDF¶
In [63]:
TFIDF = TF * IDF
In [64]:
TFIDF.head()
Out[64]:
Move things to their places¶
In [65]:
VOCAB['df'] = DF
VOCAB['idf'] = IDF
In [66]:
VOCAB.head()
Out[66]:
In [67]:
%%time
BOW['tf'] = TF.stack()
BOW['tfidf'] = TFIDF.stack()
In [68]:
BOW.head()
Out[68]:
Apply TFIDF sum to VOCAB¶
In [69]:
VOCAB['tfidf_sum'] = TFIDF.sum()
Observe results¶
In [70]:
VOCAB.sort_values('tfidf_sum', ascending=False).head(20).style.background_gradient(cmap=gradient_cmap, high=1)
Out[70]:
In [71]:
VOCAB[['term_rank','term_str','pos_max','tfidf_sum']]\
.sort_values('tfidf_sum', ascending=False).head(50)\
.style.background_gradient(cmap=gradient_cmap, high=1)
Out[71]:
In [72]:
VOCAB.loc[VOCAB.pos_max != 'NNP', ['term_rank','term_str','pos_max','tfidf_sum']]\
.sort_values('tfidf_sum', ascending=False)\
.head(50).style.background_gradient(cmap=gradient_cmap, high=1)
Out[72]:
In [73]:
BOW = BOW.join(VOCAB[['term_str','pos_max']], on='term_id')
In [74]:
BOW.sort_values('tfidf', ascending=False).head(20)\
.style.background_gradient(cmap=gradient_cmap, high=1)
Out[74]:
Visualize¶
Rank and TFIDF Sum¶
In [75]:
px.scatter(VOCAB, x='term_rank', y='tfidf_sum', hover_name='term_str', hover_data=['n'], color='pos_max')
In [76]:
px.scatter(VOCAB, x='term_rank2', y='tfidf_sum', hover_name='term_str', hover_data=['n'], color='pos_max')
Log Rank and Log TFIDF Sum¶
In [77]:
# px.scatter(VOCAB, x='term_rank', y='tfidf_sum', hover_name='term_str', hover_data=['n'], color='pos_max', log_x=True, log_y=True)
In [78]:
# px.scatter(VOCAB, x='term_rank2', y='tfidf_sum', hover_name='term_str', hover_data=['n'], color='pos_max',
# log_x=True, log_y=True)
Show Demo Table with TFIDF¶
In [79]:
demo2 = VOCAB.loc[VOCAB.term_rank.isin(rank_index), ['term_str', 'pos_max', 'term_rank', 'n', 'zipf_k', 'tfidf_sum']]
In [80]:
demo2.style.background_gradient(cmap=gradient_cmap, high=1)
Out[80]:
In [81]:
px.scatter(demo2, x='term_rank', y='tfidf_sum', log_x=True, log_y=True, text='term_str', color='pos_max', size='n')
Word-Context Matrix Entropy¶
In [82]:
WCM = DTCM / DTCM.sum()
In [83]:
WCM.sum().head()
Out[83]:
In [84]:
WCMh = WCM * np.log2(1/WCM)
In [85]:
VOCAB['h2'] = WCMh.sum()
In [86]:
VOCAB['h2'].hist();
X Factor¶
In [87]:
VOCAB['x_factor'] = np.log(VOCAB.term_rank) * VOCAB.h2
In [88]:
px.scatter(VOCAB, x='term_rank', y='x_factor', hover_name='term_str', color='pos_max', hover_data=['n'])
In [89]:
VOCAB['x_factor2'] = np.log(VOCAB.term_rank2) * VOCAB.h2
In [90]:
px.scatter(VOCAB, x='term_rank2', y='x_factor2', hover_name='term_str', color='pos_max', hover_data=['n'])
In [91]:
# px.scatter(VOCAB, x='term_rank', y='x_factor', log_x=True, log_y=True, hover_name='term_str', color='pos_max', hover_data=['n'])
In [92]:
# px.scatter(VOCAB, x='term_rank2', y='x_factor2', log_x=True, log_y=True, hover_name='term_str', color='pos_max', hover_data=['n'])
Demo Table¶
In [93]:
demo3 = VOCAB.loc[VOCAB.term_rank.isin(rank_index), ['term_str', 'pos_max', 'n', 'term_rank', 'zipf_k', 'tfidf_sum', 'h2', 'x_factor', 'term_rank2', 'x_factor2']]
In [94]:
demo3.style.background_gradient(cmap=gradient_cmap)
Out[94]:
In [95]:
# px.scatter(demo3, x='term_rank', y='x_factor', log_x=True, log_y=True, text='term_str', color='pos_max', size='n')
In [96]:
px.scatter(demo3, x='term_rank2', y='x_factor2', log_x=False, log_y=False, text='term_str', color='pos_max', size='n')
Reduce VOCAB¶
Select Significant Terms based on X Factor¶
We want to take the upper and middle segment of our graph.
In [97]:
# key_col = 'tfidf_sum'
key_col = 'x_factor2'
key_min = VOCAB[key_col].quantile(.9)
rank_min = 200
In [98]:
SIGS = VOCAB.loc[(VOCAB[key_col] >= key_min) & (VOCAB.term_rank >= rank_min)].sort_values(key_col, ascending=False)
In [99]:
SIGS.shape[0]
Out[99]:
In [100]:
SIGS[['pos_max', 'term_str', 'n', 'term_rank', 'zipf_k', 'df', 'idf', 'tfidf_sum','x_factor2']].head(100).style.background_gradient(cmap=gradient_cmap, high=1)
Out[100]:
Save Work¶
In [101]:
VOCAB.to_csv('VOCAB2.csv')
TOKEN.to_csv('TOKEN2.csv')
BOW.to_csv('DOC2.csv')
DTCM.to_csv('DTCM.csv')
TFIDF.to_csv('TFIDF.csv')
#SIGS.to_csv('SIGS.csv')
#WCM.to_csv('WCM.csv')
# BOW.to_csv('BOW.csv')
Similarity and Distance Measures¶
Config¶
We define two OHCO lists, one to match the reduced TFIDF table we are importing, and the other to define the table after we compress this table to make clustering easier.
In [102]:
OHCO_src = ['book_id', 'chap_num']
OHCO = ['book_id']
Import¶
In [103]:
import pandas as pd
import numpy as np
import re
from numpy.linalg import norm
from scipy.spatial.distance import pdist
In [104]:
import seaborn as sns
sns.set(style="ticks")
%matplotlib inline
Prepare the Data¶
Import tables¶
In [105]:
TFIDF = pd.read_csv('TFIDF.csv').set_index(OHCO_src)
VOCAB = pd.read_csv('VOCAB.csv').set_index('term_id')
LIB = pd.read_csv('LIBRARY.csv').set_index('book_id')
In [106]:
TFIDF.head()
Out[106]:
Add Some Labels to the Library¶
In [107]:
LIB.loc[LIB.book_title.str.contains('Alcott'), 'author'] = 'Alcott'
LIB['author'] = LIB['author'].fillna('Poe')
LIB['title'] = LIB.book_title.str.split(', by').apply(lambda x: x[0])
LIB.columns = ['book_id','book_title','book_file','author','title']
In [108]:
LIB
Out[108]:
Collapse TFIDF¶
We want to work with larger bags in this notebook, in order to better visualize our resulting clusters.
In [109]:
TFIDF = TFIDF.groupby(OHCO).mean()
In [110]:
TFIDF
Out[110]:
Create a DOC table¶
We want to create a new table that maps the OHCO levels to a single doc_id. We do this so that when we create a table to store pairs of docs and their distances, we can use a single-valued ID for each docs.
This table will also be used to store cluster assignments.
All of this will become clearer below!
In [111]:
DOC = TFIDF.reset_index()[OHCO] # We create a table from the OHCO in our TFIDF table
In [112]:
DOC.index.name = 'doc_id' # We give the new index a name
In [113]:
DOC.head()
Out[113]:
Add a meaningful Title to DOC index¶
In [114]:
DOC['title'] = DOC.book_id.map(LIB.author) \
+ '-' + DOC[OHCO].apply(lambda x: x.astype('str').str.cat(sep='-'), 1) \
+ ': '+ DOC.book_id.map(LIB.title)
In [115]:
DOC
Out[115]:
Create Normalized Tables¶
In [116]:
L0 = TFIDF.astype('bool').astype('int')
L1 = TFIDF.apply(lambda x: x / x.sum(), 1)
L2 = TFIDF.apply(lambda x: x / norm(x), 1)
Create Doc Pair Table¶
Create a table to store our results.
Note that pist() is a "distance matrix computation from a collection of raw observation vectors stored in a rectangular array".
In [117]:
PAIRS = pd.DataFrame(index=pd.MultiIndex.from_product([DOC.index.tolist(), DOC.index.tolist()])).reset_index()
PAIRS = PAIRS[PAIRS.level_0 < PAIRS.level_1].set_index(['level_0','level_1'])
PAIRS.index.names = ['doc_a', 'doc_b']
In [118]:
PAIRS.shape
Out[118]:
In [119]:
PAIRS.head()
Out[119]:
Compute Distances¶
In [120]:
%time PAIRS['cityblock'] = pdist(TFIDF, 'cityblock')
In [121]:
%time PAIRS['euclidean'] = pdist(TFIDF, 'euclidean')
In [122]:
%time PAIRS['cosine'] = pdist(TFIDF, 'cosine')
In [123]:
%time PAIRS['jaccard'] = pdist(L0, 'jaccard') # Fast, and similar to js
In [124]:
%time PAIRS['dice'] = pdist(L0, 'dice')
In [125]:
# %time PAIRS['js'] = pdist(L1, 'jensenshannon') # Turns out to be really slow
In [126]:
%time PAIRS['euclidean2'] = pdist(L2, 'euclidean') # Should be the same as cosine (colinear)
In [127]:
%time PAIRS['js'] = pdist(TFIDF, 'jensenshannon')
In [128]:
import scipy
scipy.__version__
Out[128]:
In [129]:
PAIRS.head()
Out[129]:
Compare Distributions¶
In [130]:
if PAIRS.shape[0] > 1000:
SAMPLE = PAIRS.sample(1000)
else:
SAMPLE = PAIRS
In [131]:
sns.pairplot(SAMPLE)
Out[131]:
In [132]:
PAIRS.sort_values('cosine').head(20).style.background_gradient('YlGn', high=1)
Out[132]:
Create Clusters¶
Hiearchical¶
In [133]:
import scipy.cluster.hierarchy as sch
import matplotlib.pyplot as plt
In [134]:
def hca(sims, linkage_method='ward', color_thresh=.3, figsize=(10, 10)):
tree = sch.linkage(sims, method=linkage_method)
labels = DOC.title.values
plt.figure()
fig, axes = plt.subplots(figsize=figsize)
dendrogram = sch.dendrogram(tree,
labels=labels,
orientation="left",
count_sort=True,
distance_sort=True,
above_threshold_color='.75',
color_threshold=color_thresh
)
plt.tick_params(axis='both', which='major', labelsize=14)
In [135]:
hca(PAIRS.cosine, color_thresh=1)
In [136]:
hca(PAIRS.jaccard, color_thresh=.6)
In [137]:
hca(PAIRS.euclidean, color_thresh=.3)
In [138]:
hca(PAIRS.cityblock, color_thresh=8)
In [139]:
hca(PAIRS.js, color_thresh=.6)
K-Means¶
K-Means only uses Euclidean distance. Why?
- The K-Means procedure does not explicitly use pairwise distances between data points.
- Instead, it repeatedly assigns points to the closest centroid thereby using Euclidean distance from data points to a centroid.
- However, K-Means is implicitly based on pairwise Euclidean distances between data points, because the sum of squared deviations from centroid is equal to the sum of pairwise squared Euclidean distances divided by the number of points.
- The term "centroid" is itself from Euclidean geometry. It is multivariate mean in Euclidean space. Euclidean space is about euclidean distances. Non-Euclidean distances will generally not span Euclidean space. That's why K-Means is for Euclidean distances only.
See the Cross Validated post on this.
In [140]:
from sklearn.cluster import KMeans
In [141]:
n_clusters = 4
In [142]:
DOC['y_raw'] = KMeans(n_clusters).fit_predict(TFIDF)
DOC['y_L0'] = KMeans(n_clusters).fit_predict(L0)
DOC['y_L1'] = KMeans(n_clusters).fit_predict(L1)
DOC['y_L2'] = KMeans(n_clusters).fit_predict(L2)
In [143]:
DOC.sort_values('y_raw').style.background_gradient(cmap='YlGn', high=1)
Out[143]:
Save Work¶
In [144]:
DOC.to_csv('DOC3.csv')
LIB.to_csv('LIB3.csv')
TFIDF.to_csv('TFIDF_book_final.csv')
Principal Component Analysis¶
In [37]:
OHCO = ['book_id', 'chap_num']
In [456]:
import pandas as pd
import numpy as np
from sklearn.decomposition import PCA
from scipy.linalg import norm
In [457]:
import plotly_express as px
import seaborn as sns
In [458]:
sns.set(style='ticks')
%matplotlib inline
In [38]:
LIB = pd.read_csv('LIB3.csv').set_index('book_id')
VOCAB = pd.read_csv('VOCAB2.csv').set_index('term_id')
TFIDF = pd.read_csv('TFIDF.csv').set_index(OHCO)
In [150]:
TFIDF.shape
Out[150]:
Normalize doc vector lengths¶
We use L2 normalization
In [151]:
TFIDF = TFIDF.apply(lambda x: x / np.sqrt(np.square(x).sum()), 1)
Normalize term vector variance¶
We do not normalize variance, which we would normally do, such as with data containing divergent units of measure. \ This is because to do so would exaggerate the importance of rare words (see Ng, 2008: 6m40s — 8m00s).
Center the word vectors¶
Note that we are taking the column-wise means -- the means for the term vectors. \ We don't really need to do this. But it is typical for PCA. \ NOTE: Some argue that centering alters the cosine angles.
In [152]:
TFIDF = TFIDF - TFIDF.mean()
Compute Covariance Matrix¶
$n = |X| = |Y|$
$Cov(X,Y) = \dfrac{\sum_{i=1}^{n} (x_i - \mu_X) (y_i - \mu_Y)}{n - 1} = \dfrac{XY}{n-1}$
We could we use the built in Pandas method here, but compute it ourselves.
In [153]:
# COV = TFIDF.cov() # This also centers the vectors
COV = TFIDF.T.dot(TFIDF) / (TFIDF.shape[0] - 1)
In [154]:
COV.head()
Out[154]:
In [155]:
COV.iloc[:5,:10].style.background_gradient()
Out[155]:
Decompose the Matrix¶
There a at least three options to choose from. We go with SciPy's Hermitian Eigendecomposition \
method eigh(), since our covarience matrix is symmetric.
In [156]:
from scipy.linalg import eigh
In [157]:
%time eig_vals, eig_vecs = eigh(COV)
Convert eigen data to dataframes¶
In [158]:
TERM_IDX = COV.index
In [159]:
EIG_VEC = pd.DataFrame(eig_vecs, index=TERM_IDX, columns=TERM_IDX)
In [160]:
EIG_VAL = pd.DataFrame(eig_vals, index=TERM_IDX, columns=['eig_val'])
EIG_VAL.index.name = 'term_id'
In [161]:
EIG_VEC.iloc[:5, :10].style.background_gradient()
Out[161]:
In [162]:
EIG_VAL.iloc[:5]
Out[162]:
Select Principal Components¶
Next, we associate each eigenvalue with its corresponding column in the eigenvalue matrix. \
This is why we transpose the EIG_VEC dataframe.
Combine eigenvalues and eignvectors¶
In [163]:
EIG_PAIRS = EIG_VAL.join(EIG_VEC.T)
In [164]:
EIG_PAIRS.head()
Out[164]:
Next, we sort in descending order and pick the top K (=10).
Compute and Show Explained Variance¶
We might have usd this value to sort our components.
In [165]:
EIG_PAIRS['exp_var'] = np.round((EIG_PAIRS.eig_val / EIG_PAIRS.eig_val.sum()) * 100, 2)
In [166]:
EIG_PAIRS.exp_var.sort_values(ascending=False).head().plot.bar(rot=45)
Out[166]:
Pick Top K (10) Components¶
We pick these based on explained variance.
In [167]:
COMPS = EIG_PAIRS.sort_values('exp_var', ascending=False).head(10).reset_index(drop=True)
COMPS.index.name = 'comp_id'
COMPS.index = ["PC{}".format(i) for i in COMPS.index.tolist()]
In [168]:
COMPS
Out[168]:
Inspect terms associated with eigenvectors¶
In [169]:
VOCAB.loc[[int(x) for x in EIG_PAIRS.sort_values('exp_var', ascending=False).head(10).index], 'term_str']
Out[169]:
Show Loadings¶
Loadings sow the contribution of each term to the component. \ We'll just look at the topi 10 words for the first two components in the Book version.
In [170]:
LOADINGS = COMPS[TERM_IDX].T
LOADINGS.index.name = 'term_id'
In [248]:
LOADINGS.head().style.background_gradient()
Out[248]:
In [172]:
LOADINGS['term_str'] = LOADINGS.apply(lambda x: VOCAB.loc[int(x.name)].term_str, 1)
In [173]:
# l0_pos = LOADINGS.sort_values('PC0', ascending=True).head(10).term_str.str.cat(sep=' ')
# l0_neg = LOADINGS.sort_values('PC0', ascending=False).head(10).term_str.str.cat(sep=' ')
# l1_pos = LOADINGS.sort_values('PC1', ascending=True).head(10).term_str.str.cat(sep=' ')
# l1_neg = LOADINGS.sort_values('PC1', ascending=False).head(10).term_str.str.cat(sep=' ')
In [174]:
# print('Books PC0+', l0_pos)
# print('Books PC0-', l0_neg)
# print('Books PC1+', l1_pos)
# print('Books PC1-', l1_neg)
Project Docs onto New Subspace¶
We get the dot product of the DTM matrix and the new component matrix, which we will call DCM -- for document-component matrix. \ This has the effect of replacing the features of the DTM with the features of the transposed component matrix.
In [175]:
DCM = TFIDF.dot(COMPS[TERM_IDX].T)
In [176]:
DCM
Out[176]:
We add metadata to our new, reduced matrices for display purposes.
In [177]:
DCM = DCM.join(LIB[['author','title']], on='book_id')
In [178]:
DCM['doc'] = DCM.apply(lambda x: "{}-{}-{}".format(x.author, x.title, x.name[1]), 1)
In [179]:
DCM.head().style.background_gradient()
Out[179]:
Visualize¶
In [180]:
def vis_pcs(M, a, b, label='author', prefix='PC'):
fig = px.scatter(M, prefix + str(a), prefix + str(b),
color=label,
hover_name='doc', marginal_x='box')
fig.show()
PC 0 and 1¶
In [181]:
vis_pcs(DCM, 0, 1)
In [182]:
vis_pcs(DCM, 0, 1, label='title')
PC 1 and 2¶
In [183]:
vis_pcs(DCM, 1, 2)
In [184]:
vis_pcs(DCM, 1, 2, label='title')
PC 2 and 3¶
In [185]:
vis_pcs(DCM, 2, 3)
In [186]:
vis_pcs(DCM, 2, 3, label='title')
PC 3 and 4¶
In [187]:
vis_pcs(DCM, 3, 4, label='author')
In [188]:
vis_pcs(DCM, 3, 4, label='title')
Compare to Scikit Learn¶
In [189]:
pca_engine = PCA(n_components=10)
In [190]:
DCM_sk = pd.DataFrame(pca_engine.fit_transform(TFIDF), index=TFIDF.index)
DCM_sk.columns = ['PC{}'.format(i) for i in DCM_sk.columns]
DCM_sk = DCM_sk.join(LIB[['author','title']], on='book_id')
DCM_sk['doc'] = DCM_sk.apply(lambda x: "{}-{}-{}".format(x.author, x.title, x.name[1]), 1)
In [191]:
DCM_sk.head().style.background_gradient()
Out[191]:
PC 0 and 1¶
In [192]:
vis_pcs(DCM_sk, 0, 1)
PC 0 and 1¶
In [193]:
vis_pcs(DCM_sk, 0, 1)
In [194]:
px.scatter_3d(DCM_sk, 'PC0', 'PC1','PC2', color='title', hover_name='doc', height=1000, width=1200)
Show Loadings¶
In [195]:
LOADINGS_sk = pd.DataFrame(pca_engine.components_.T * np.sqrt(pca_engine.explained_variance_))
LOADINGS_sk.columns = ["PC{}".format(i) for i in LOADINGS_sk.columns]
In [196]:
LOADINGS_sk.index = TFIDF.columns
LOADINGS_sk.index.name = 'term_id'
LOADINGS_sk['term_str'] = LOADINGS_sk.apply(lambda x: VOCAB.loc[int(x.name)].term_str, 1)
In [197]:
# pc0_pos = LOADINGS_sk.sort_values('PC0', ascending=False).head(10).term_str.str.cat(sep=' ')
# pc0_neg = LOADINGS_sk.sort_values('PC0', ascending=True).head(10).term_str.str.cat(sep=' ')
# pc1_pos = LOADINGS_sk.sort_values('PC1', ascending=False).head(10).term_str.str.cat(sep=' ')
# pc1_neg = LOADINGS_sk.sort_values('PC1', ascending=True).head(10).term_str.str.cat(sep=' ')
In [198]:
# print('BOOKS PC0+', pc0_pos)
# print('BOOKS PC0-', pc0_neg)
# print('BOOKS PC1+', pc1_pos)
# print('BOOKS PC1-', pc1_neg)
SAVE¶
In [232]:
DCM.to_csv('PCA_DCM_books.csv')
COMPS.to_csv('PCA_TCM_books.csv')
LOADINGS.to_csv('LOADINGS.csv')
Topic Models (LDA)¶
In [3]:
n_terms = 4000
n_topics = 30
# raf - do minimum of 20
max_iter = 5
OHCO = ['book_id', 'chap_num', 'para_num']
Imports¶
In [4]:
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation as LDA
In [5]:
%matplotlib inline
Prepare Data¶
Convert TOKENS to table of paragraphs¶
Scikit Learn wants an F1 style corpus. We create onefrom our annotated TOKEN table, keeping only regular nouns.
In [6]:
TOKENS = pd.read_csv('TOKEN2.csv')
In [7]:
TOKENS.head()
Out[7]:
In [8]:
PARAS = TOKENS[TOKENS.pos.str.match(r'^NNS?$')]\
.groupby(OHCO).term_str\
.apply(lambda x: ' '.join(x))\
.to_frame()\
.rename(columns={'term_str':'para_str'})
In [9]:
PARAS.head()
Out[9]:
Create Vector Space¶
We use Scikit Learn's CountVectorizer to convert our F1 corpus of paragraphs into a document-term vector space of word counts.
In [10]:
tfv = CountVectorizer(max_features=n_terms, stop_words='english')
tf = tfv.fit_transform(PARAS.para_str)
TERMS = tfv.get_feature_names()
Generate Model¶
We run Scikit Learn's LatentDirichletAllocation algorithm and extract the THETA and PHI tables.
In [11]:
lda = LDA(n_components=n_topics, max_iter=max_iter, learning_offset=50., random_state=0)
THETA¶
In [12]:
THETA = pd.DataFrame(lda.fit_transform(tf), index=PARAS.index)
THETA.columns.name = 'topic_id'
In [13]:
THETA.sample(20).style.background_gradient()
Out[13]:
PHI¶
In [14]:
PHI = pd.DataFrame(lda.components_, columns=TERMS)
PHI.index.name = 'topic_id'
PHI.columns.name = 'term_str'
In [15]:
PHI.T.head().style.background_gradient()
Out[15]:
Inspect Results¶
Get Top Terms per Topic¶
In [16]:
TOPICS = PHI.stack().to_frame().rename(columns={0:'weight'})\
.groupby('topic_id')\
.apply(lambda x:
x.weight.sort_values(ascending=False)\
.head(10)\
.reset_index()\
.drop('topic_id',1)\
.term_str)
In [17]:
TOPICS
Out[17]:
In [18]:
TOPICS['label'] = TOPICS.apply(lambda x: str(x.name) + ' ' + ' '.join(x), 1)
Sort Topics by Doc Weight¶
In [19]:
TOPICS['doc_weight_sum'] = THETA.sum()
In [20]:
TOPICS.sort_values('doc_weight_sum', ascending=True).plot.barh(y='doc_weight_sum', x='label', figsize=(5,10))
Out[20]:
Explore Topics by Author¶
In [21]:
LIB = pd.read_csv('LIB3.csv').set_index('book_id')
In [22]:
topic_cols = [t for t in range(n_topics)]
AUTHORS = THETA.join(LIB, on='book_id')\
.reset_index().set_index(['author']+OHCO)\
.groupby('author')[topic_cols].mean()\
.T
AUTHORS.index.name = 'topic_id'
In [23]:
AUTHORS['topterms'] = TOPICS[[i for i in range(10)]].apply(lambda x: ' '.join(x), 1)
In [24]:
AUTHORS.sort_values('Alcott', ascending=False).style.background_gradient()
Out[24]:
In [25]:
AUTHORS.sort_values('Poe', ascending=False).style.background_gradient()
Out[25]:
In [26]:
import plotly_express as px
In [27]:
# px.scatter(AUTHORS.reset_index(), 'Alcott', 'Poe', hover_name='topterms', text='topic_id')\
# .update_traces(mode='text')
Clutser Topics¶
In [28]:
import scipy.cluster.hierarchy as sch
from scipy.spatial.distance import pdist
from sklearn.cluster import AgglomerativeClustering
from sklearn.preprocessing import normalize
import matplotlib.pyplot as plt
In [29]:
def plot_tree(tree, labels):
plt.figure()
fig, axes = plt.subplots(figsize=(5, 10))
dendrogram = sch.dendrogram(tree, labels=labels, orientation="left")
plt.tick_params(axis='both', which='major', labelsize=14)
In [30]:
SIMS = pdist(normalize(PHI), metric='euclidean')
TREE = sch.linkage(SIMS, method='ward')
In [31]:
labels = ["{}: {}".format(a,b) for a, b in zip(AUTHORS.index, AUTHORS.topterms.tolist())]
In [32]:
plot_tree(TREE, labels)
LDA Viz¶
In [33]:
import pyLDAvis
import pyLDAvis.sklearn
pyLDAvis.enable_notebook()
In [34]:
pyLDAvis.sklearn.prepare(lda, tf, tfv)
Out[34]:
In [35]:
PHI.T
Out[35]:
In [39]:
VOCAB
Out[39]:
In [40]:
topics_terms = pd.concat([VOCAB.reset_index().set_index('term_str'), PHI.T], join='inner', axis=1)\
.reset_index().set_index('term_id')
In [41]:
topics_terms
Out[41]:
In [42]:
THETA.to_csv('LDA_docs_topics.csv')
topics_terms.to_csv('LDA_topics_terms.csv')
Word Embeddings (word2vec)¶
In [460]:
OHCO = ['book_id', 'chap_num', 'para_num', 'sent_num', 'token_num']
BAG = OHCO[:4] # Paragraphs
# BAG = OHCO[:5] # Sentences
window = 5
Imports¶
In [461]:
from gensim.models import word2vec
from sklearn.manifold import TSNE
import plotly_express as px
In [462]:
%matplotlib inline
Process¶
Import TOKENS and convert to a corpus for Gensim¶
We import data from the TOKEN table of the novels corpus, excluding proper nouns.
In [463]:
TOKENS = pd.read_csv('TOKEN2.csv').set_index(OHCO)
In [464]:
corpus = TOKENS[~TOKENS.pos.str.match('NNPS?')]\
.groupby(BAG)\
.term_str.apply(lambda x: x.tolist())\
.reset_index()['term_str'].tolist()
Generate word embeddings with Gensim's library¶
In [465]:
model = word2vec.Word2Vec(corpus, size=246, window=window, min_count=200, workers=4)
Visualize with tSNE¶
Generate coordinates to plot¶
In [482]:
coords = pd.DataFrame(index=range(len(model.wv.vocab)))
coords['label'] = [w for w in model.wv.vocab]
coords['vector'] = coords['label'].apply(lambda x: model.wv.get_vector(x))
In [483]:
coords.head()
Out[483]:
Use ScikitLearn's TSNE library¶
In [484]:
tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)
tsne_values = tsne_model.fit_transform(coords['vector'].tolist())
In [485]:
coords['x'] = tsne_values[:,0]
coords['y'] = tsne_values[:,1]
In [486]:
coords.head()
Out[486]:
Plot the coordinates¶
In [487]:
px.scatter(coords, 'x', 'y', text='label', height=1000).update_traces(mode='text')
In [488]:
coords = coords.rename(columns={'label':'term_str'})
coords = coords[['term_str','vector']]
coords = coords.set_index('term_str')
coords.head()
Out[488]:
In [489]:
EMBEDDINGS = pd.concat([VOCAB.reset_index().set_index('term_str'), coords], join='inner', axis=1)\
.reset_index().set_index('term_id')
In [492]:
EMBEDDINGS.head()
Out[492]:
In [493]:
EMBEDDINGS.to_csv('Term_Embeddings.csv')
Semantic Algebra¶
Analogy¶
$A : B :: C : D? \rightarrow B - A + C = D$
In [264]:
def complete_analogy(A, B, C, n=2):
try:
return model.wv.most_similar(positive=[B, C], negative=[A])[0:n]
except KeyError as e:
print('Error:', e)
return None
In [265]:
complete_analogy('i', 'we', 'you')
Out[265]:
In [266]:
complete_analogy('man', 'woman', 'he')
Out[266]:
In [267]:
complete_analogy('man','boy','woman')
Out[267]:
Sentiment Analysis¶
In [380]:
# set up
#data_dir = 'data/'
novels_csv = 'TOKEN2.csv'
vocab_csv = 'VOCAB2.csv'
lib_csv = 'LIB3.csv'
bow_csv = 'BOW.csv'
# For TOKENS
OHCO = ['book_id', 'chap_num', 'para_num', 'sent_num', 'token_num']
BOOKS = OHCO[:1]
CHAPS = OHCO[:2]
PARAS = OHCO[:3]
SENTS = OHCO[:4]
salex_csv = 'salex_nrc.csv'
emo_cols = "anger anticipation disgust fear joy sadness surprise trust polarity".split()
In [381]:
import pandas as pd
import numpy as np
import seaborn as sns
import plotly_express as px
from IPython.core.display import display, HTML
In [382]:
sns.set()
%matplotlib inline
Acquire Data¶
In [383]:
TOKENS = pd.read_csv(novels_csv).set_index(OHCO)
VOCAB = pd.read_csv(vocab_csv).set_index('term_id')
LIB = pd.read_csv(lib_csv).set_index('book_id')
BOW = pd.read_csv(bow_csv)
# Add token_str to BOW
BOW['term_str'] = BOW.term_id.map(VOCAB.term_str)
BOW = BOW.set_index(['book_id','chap_num','term_id'])
In [384]:
BOW.sort_index()
Out[384]:
In [385]:
LIB
Out[385]:
In [386]:
# book name and id
old = 2787
eight = 2726
jackjill = 2786
jo = 3499
littleM = 2788
littleW = 514
rose = 2804
under = 3795
poe1 = 1
poe2 = 2
poe3 = 3
poe4 = 4
poe5 = 5
Get SA Lexicon¶
In [387]:
salex_csv = 'salex_nrc.csv'
In [388]:
SALEX = pd.read_csv(salex_csv).set_index('term_str')
SALEX.columns = [col.replace('nrc_','') for col in SALEX.columns]
SALEX['polarity'] = SALEX.positive - SALEX.negative
In [389]:
SALEX
Out[389]:
Compute Sentiment¶
Combine SALEX with VOCAB¶
In [391]:
VOCAB.head()
Out[391]:
In [392]:
len(VOCAB)
Out[392]:
In [402]:
V = pd.concat([VOCAB.reset_index().set_index('term_str'), SALEX], join='inner', axis=1)\
.reset_index().set_index('term_id')
In [403]:
V.head()
Out[403]:
In [404]:
len(V)
Out[404]:
In [405]:
V.to_csv('Vocab_Sentiment.csv')
Combine VOCAB with BOW¶
In [406]:
BOWIDX = BOW.index
B = BOW.merge(V, on='term_str', how='left')
B.index = BOWIDX
B = B.dropna()
# didnt have tf, tfidf columns
bcols = ['n_x', 'term_str', 'term_rank', 'n_y', 'num', 'pos_max', 'term_rank2', 'p',
'tfidf_sum', 'anger', 'anticipation', 'disgust', 'fear', 'joy', 'negative', 'positive', 'sadness', 'surprise', 'trust', 'polarity']
B = B[bcols]
for col in emo_cols:
B[col] = B[col] * B.tfidf_sum # used to be B.tfidf
In [407]:
B.head()
Out[407]:
In [408]:
B.tfidf_sum
Out[408]:
In [409]:
B.to_csv('Document_Sentiment.csv')
Explore Emotion Words and POS¶
In [292]:
B.pos_max.value_counts().sort_values().plot.barh()
Out[292]:
In [293]:
EMO_BOOKS = B.groupby(['book_id'])[emo_cols].mean()
EMO_CHAPS = B.groupby(['book_id','chap_num'])[emo_cols].mean()
In [294]:
EMO_BOOKS.index = LIB.book_title
In [295]:
EMO_BOOKS.plot.bar(figsize=(25,10))
Out[295]:
Compare Texts¶
In [296]:
EIGHT = EMO_CHAPS.loc[eight].copy()
JACKJILL = EMO_CHAPS.loc[jackjill].copy()
OLD = EMO_CHAPS.loc[old].copy()
JO = EMO_CHAPS.loc[jo].copy()
LITTLEM = EMO_CHAPS.loc[littleM].copy()
LITTLEW = EMO_CHAPS.loc[littleW].copy()
ROSE = EMO_CHAPS.loc[rose].copy()
UNDER = EMO_CHAPS.loc[under].copy()
POE1 = EMO_CHAPS.loc[poe1].copy()
POE2 = EMO_CHAPS.loc[poe2].copy()
POE3 = EMO_CHAPS.loc[poe3].copy()
POE4 = EMO_CHAPS.loc[poe4].copy()
POE5 = EMO_CHAPS.loc[poe5].copy()
In [297]:
EIGHT.mean().sort_values().plot.barh();
In [298]:
JACKJILL.mean().sort_values().plot.barh();
In [299]:
OLD.mean().sort_values().plot.barh();
In [300]:
JO.mean().sort_values().plot.barh();
In [301]:
LITTLEM.mean().sort_values().plot.barh();
In [302]:
LITTLEW.mean().sort_values().plot.barh();
In [303]:
ROSE.mean().sort_values().plot.barh();
In [304]:
UNDER.mean().sort_values().plot.barh();
In [305]:
POE1.mean().sort_values().plot.barh();
In [306]:
POE2.mean().sort_values().plot.barh();
In [307]:
POE3.mean().sort_values().plot.barh();
In [308]:
POE4.mean().sort_values().plot.barh();
In [309]:
POE5.mean().sort_values().plot.barh();
What is Trust?¶
In [310]:
trust_words = pd.Series(SALEX[SALEX['trust'] == 1].index.tolist())
In [311]:
trust_words.sample(10)
Out[311]:
Mean Sentiment by Chapter¶
In [312]:
EIGHT_chaps = EMO_CHAPS.loc[eight]
JACKJILL_chaps = EMO_CHAPS.loc[jackjill]
OLD_chaps = EMO_CHAPS.loc[old]
JO_chaps = EMO_CHAPS.loc[jo]
LITTLEM_chaps = EMO_CHAPS.loc[littleM]
LITTLEW_chaps = EMO_CHAPS.loc[littleW]
ROSE_chaps = EMO_CHAPS.loc[rose]
UNDER_chaps = EMO_CHAPS.loc[under]
POE1_chaps = EMO_CHAPS.loc[poe1]
POE2_chaps = EMO_CHAPS.loc[poe2]
POE3_chaps = EMO_CHAPS.loc[poe3]
POE4_chaps = EMO_CHAPS.loc[poe4]
POE5_chaps = EMO_CHAPS.loc[poe5]
In [313]:
EIGHT_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[313]:
In [314]:
JACKJILL_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[314]:
In [315]:
OLD_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[315]:
In [316]:
JO_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[316]:
In [317]:
LITTLEM_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[317]:
In [318]:
LITTLEW_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[318]:
In [319]:
ROSE_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[319]:
In [320]:
UNDER_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[320]:
In [321]:
POE1_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[321]:
In [322]:
POE2_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[322]:
In [323]:
POE3_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[323]:
In [324]:
POE4_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[324]:
In [325]:
POE5_chaps.style.background_gradient(cmap='YlGn', high=.25)
Out[325]:
Create Stacked Dataframes (for Plotly Express)¶
In [326]:
EIGHT_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(eight))
JACKJILL_chaps_thin= EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(jackjill))
OLD_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(old))
JO_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(jo))
LITTLEM_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(littleM))
LITTLEW_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(littleW))
ROSE_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(rose))
UNDER_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(under))
POE1_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(poe1))
POE2_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(poe2))
POE3_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(poe3))
POE4_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(poe4))
POE5_chaps_thin = EMO_CHAPS.stack().to_frame().reset_index().rename(columns={0:'value','level_2':'emo'}).query("book_id == {}".format(poe5))
In [327]:
def plot_sentiments(df, emo='polarity'):
FIG = dict(figsize=(25, 5), legend=True, fontsize=14, rot=45)
df[emo].plot(**FIG)
In [328]:
# plot_sentiments(PERSUASION_chaps, emo_cols)
px.line(EIGHT_chaps_thin, x='chap_num', y='value', color='emo')
In [329]:
# plot_sentiments(MOBYDICK_chaps, emo_cols)
px.line(JACKJILL_chaps_thin, x='chap_num', y='value', color='emo')
In [330]:
px.line(OLD_chaps_thin, x='chap_num', y='value', color='emo')
In [331]:
px.line(JO_chaps_thin, x='chap_num', y='value', color='emo')
In [332]:
px.line(LITTLEM_chaps_thin, x='chap_num', y='value', color='emo')
In [333]:
px.line(LITTLEW_chaps_thin, x='chap_num', y='value', color='emo')
In [334]:
px.line(ROSE_chaps_thin, x='chap_num', y='value', color='emo')
In [335]:
px.line(UNDER_chaps_thin, x='chap_num', y='value', color='emo')
In [336]:
px.line(POE1_chaps_thin, x='chap_num', y='value', color='emo')
In [337]:
px.line(POE2_chaps_thin, x='chap_num', y='value', color='emo')
In [338]:
px.line(POE3_chaps_thin, x='chap_num', y='value', color='emo')
In [339]:
px.line(POE4_chaps_thin, x='chap_num', y='value', color='emo')
In [340]:
px.line(POE5_chaps_thin, x='chap_num', y='value', color='emo')
Cose Read Sentiment in Texts¶
Combine VOCAB + SALEX with TOKENS¶
We need to do this to reconstruct the sentences, which are lost in the BOW representation.
In [341]:
TOKENSIDX = TOKENS.index
T = TOKENS.merge(V, on='term_str', how='left')
T.index = TOKENSIDX
T = T.fillna(0)
In [342]:
EIGHT2 = T.loc[eight].copy()
JACKJILL2 = T.loc[jackjill].copy()
OLD2 = T.loc[old].copy()
JO2 = T.loc[jo].copy()
LITTLEM2 = T.loc[littleM].copy()
LITTLEW2 = T.loc[littleW].copy()
ROSE2 = T.loc[rose].copy()
UNDER2 = T.loc[under].copy()
POE12 = T.loc[poe1].copy()
POE22 = T.loc[poe2].copy()
POE32 = T.loc[poe3].copy()
POE42 = T.loc[poe4].copy()
POE52 = T.loc[poe5].copy()
In [343]:
emo = 'polarity'
In [344]:
EIGHT2['html'] = EIGHT2.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
JACKJILL2['html'] = JACKJILL2.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
OLD2['html'] = OLD2.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
JO2['html'] = JO2.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
LITTLEM2['html'] = LITTLEM2.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
LITTLEW2['html'] = LITTLEW2.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
ROSE2['html'] = ROSE2.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
UNDER2['html'] = UNDER2.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
POE12['html'] = POE12.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
POE22['html'] = POE22.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
POE32['html'] = POE32.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
POE42['html'] = POE42.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
POE52['html'] = POE52.apply(lambda x: "<span class='sent{}'>{}</span>".format(int(np.sign(x[emo])), x.term_str), 1)
In [345]:
EIGHT2['html'].sample(10)
Out[345]:
In [346]:
EIGHT2
Out[346]:
In [347]:
EIGHT2_sents = EIGHT2.groupby(SENTS[1:])[emo_cols].mean()
JACKJILL2_sents = JACKJILL2.groupby(SENTS[1:])[emo_cols].mean()
OLD2_sents = OLD2.groupby(SENTS[1:])[emo_cols].mean()
JO2_sents = JO2.groupby(SENTS[1:])[emo_cols].mean()
LITTLEM2_sents = LITTLEM2.groupby(SENTS[1:])[emo_cols].mean()
LITTLEW2_sents = LITTLEW2.groupby(SENTS[1:])[emo_cols].mean()
ROSE2_sents = ROSE2.groupby(SENTS[1:])[emo_cols].mean()
UNDER2_sents = UNDER2.groupby(SENTS[1:])[emo_cols].mean()
POE12_sents = POE12.groupby(SENTS[1:])[emo_cols].mean()
POE22_sents = POE22.groupby(SENTS[1:])[emo_cols].mean()
POE32_sents = POE32.groupby(SENTS[1:])[emo_cols].mean()
POE42_sents = POE42.groupby(SENTS[1:])[emo_cols].mean()
POE52_sents = POE52.groupby(SENTS[1:])[emo_cols].mean()
In [348]:
EIGHT2_sents['sent_str'] = EIGHT2.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
EIGHT2['html_str'] = EIGHT2.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [349]:
JACKJILL2_sents['sent_str'] = JACKJILL2.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
JACKJILL2_sents['html_str'] = JACKJILL2.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [350]:
OLD2_sents['sent_str'] = OLD2.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
OLD2_sents['html_str'] = OLD2.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [351]:
JO2_sents['sent_str'] = JO2.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
JO2_sents['html_str'] = JO2.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [352]:
LITTLEM2_sents['sent_str'] = LITTLEM2.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
LITTLEM2_sents['html_str'] = LITTLEM2.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [353]:
LITTLEW2_sents['sent_str'] = LITTLEW2.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
LITTLEW2_sents['html_str'] = LITTLEW2.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [354]:
ROSE2_sents['sent_str'] = ROSE2.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
ROSE2_sents['html_str'] = ROSE2.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [355]:
UNDER2_sents['sent_str'] = UNDER2.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
UNDER2_sents['html_str'] = UNDER2.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [356]:
POE12_sents['sent_str'] = POE12.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
POE12_sents['html_str'] = POE12.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [357]:
POE22_sents['sent_str'] = POE22.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
POE22_sents['html_str'] = POE22.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [358]:
POE32_sents['sent_str'] = POE32.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
POE32_sents['html_str'] = POE32.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [359]:
POE42_sents['sent_str'] = POE42.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
POE42_sents['html_str'] = POE42.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [360]:
POE52_sents['sent_str'] = POE52.groupby(SENTS[1:]).term_str.apply(lambda x: x.str.cat(sep=' '))
POE52_sents['html_str'] = POE52.groupby(SENTS[1:]).html.apply(lambda x: x.str.cat(sep=' '))
In [361]:
def sample_sentences(df):
rows = []
for idx in df.sample(10).index:
valence = round(df.loc[idx, emo], 4)
t = 0
if valence > t: color = '#ccffcc'
elif valence < t: color = '#ffcccc'
else: color = '#f2f2f2'
z=0
rows.append("""<tr style="background-color:{0};padding:.5rem 1rem;font-size:110%;">
<td>{1}</td><td>{3}</td><td width="400" style="text-align:left;">{2}</td>
</tr>""".format(color, valence, df.loc[idx, 'html_str'], idx))
display(HTML('<style>#sample1 td{font-size:120%;vertical-align:top;} .sent-1{color:red;font-weight:bold;} .sent1{color:green;font-weight:bold;}</style>'))
display(HTML('<table id="sample1"><tr><th>Sentiment</th><th>ID</th><th width="600">Sentence</th></tr>'+''.join(rows)+'</table>'))
In [362]:
# sample_sentences(EIGHT2_sents)
In [363]:
# sample_sentences(JACKJILL2_sents)
Try VADER¶
In [364]:
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyser = SentimentIntensityAnalyzer()
In [365]:
EIGHT_vader_cols = EIGHT2_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
EIGHT_vader = pd.concat([EIGHT2_sents, EIGHT_vader_cols], axis=1)
In [366]:
w = int(EIGHT_vader.shape[0] / 5)
EIGHT_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
EIGHT_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
EIGHT_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[366]:
In [367]:
JACKJILL_vader_cols = JACKJILL2_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
JACKJILL_vader = pd.concat([JACKJILL2_sents, JACKJILL_vader_cols], axis=1)
In [368]:
w = int(JACKJILL_vader.shape[0] / 5)
JACKJILL_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
JACKJILL_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
JACKJILL_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[368]:
In [369]:
OLD_vader_cols = OLD2_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
OLD_vader = pd.concat([OLD2_sents, OLD_vader_cols], axis=1)
w = int(OLD_vader.shape[0] / 5)
OLD_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
OLD_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
OLD_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[369]:
In [370]:
JO_vader_cols = JO2_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
JO_vader = pd.concat([JO2_sents, JO_vader_cols], axis=1)
w = int(JO_vader.shape[0] / 5)
JO_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
JO_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
JO_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[370]:
In [371]:
LITTLEM_vader_cols = LITTLEM2_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
LITTLEM_vader = pd.concat([LITTLEM2_sents, LITTLEM_vader_cols], axis=1)
w = int(LITTLEM_vader.shape[0] / 5)
LITTLEM_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
LITTLEM_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
LITTLEM_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[371]:
In [372]:
LITTLEW_vader_cols = LITTLEW2_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
LITTLEW_vader = pd.concat([LITTLEW2_sents, LITTLEW_vader_cols], axis=1)
w = int(LITTLEW_vader.shape[0] / 5)
LITTLEW_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
LITTLEW_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
LITTLEW_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[372]:
In [373]:
ROSE_vader_cols = ROSE2_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
ROSE_vader = pd.concat([ROSE2_sents, ROSE_vader_cols], axis=1)
w = int(ROSE_vader.shape[0] / 5)
ROSE_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
ROSE_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
ROSE_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[373]:
In [374]:
UNDER_vader_cols = UNDER2_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
UNDER_vader = pd.concat([UNDER2_sents, UNDER_vader_cols], axis=1)
w = int(UNDER_vader.shape[0] / 5)
UNDER_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
UNDER_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
UNDER_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[374]:
In [375]:
POE1_vader_cols = POE12_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
POE1_vader = pd.concat([POE12_sents, POE1_vader_cols], axis=1)
w = int(POE1_vader.shape[0] / 5)
POE1_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
POE1_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
POE1_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[375]:
In [376]:
POE2_vader_cols = POE22_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
POE2_vader = pd.concat([POE22_sents, POE2_vader_cols], axis=1)
w = int(POE2_vader.shape[0] / 5)
POE2_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
POE2_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
POE2_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[376]:
In [377]:
POE3_vader_cols = POE32_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
POE3_vader = pd.concat([POE32_sents, POE3_vader_cols], axis=1)
w = int(POE3_vader.shape[0] / 5)
POE3_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
POE3_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
POE3_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[377]:
In [378]:
POE4_vader_cols = POE42_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
POE4_vader = pd.concat([POE42_sents, POE4_vader_cols], axis=1)
w = int(POE4_vader.shape[0] / 5)
POE4_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
POE4_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
POE4_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[378]:
In [379]:
POE5_vader_cols = POE52_sents.sent_str.apply(analyser.polarity_scores).apply(lambda x: pd.Series(x))
POE5_vader = pd.concat([POE52_sents, POE5_vader_cols], axis=1)
w = int(POE5_vader.shape[0] / 5)
POE5_vader[['pos','neg']].rolling(w).mean().plot(figsize=(25,5))
POE5_vader[['neu']].rolling(w).mean().plot(figsize=(25,5))
POE5_vader[['compound']].rolling(w).mean().plot(figsize=(25,5))
Out[379]:
In [ ]: